{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "phonemeRecognitionDNNHMM.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BadrOuannas/DD2476-Project/blob/master/phonemeRecognitionDNNHMM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QY8fnnBnRvr",
        "colab_type": "text"
      },
      "source": [
        "### Import python libraries and packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pLTFXLdRnb6x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import scipy.io.wavfile as scipywav\n",
        "import scipy as scipy\n",
        "import scipy.signal as signal\n",
        "import math\n",
        "import unittest\n",
        "import matplotlib.pyplot as plt\n",
        "from scipy.fftpack import fft\n",
        "import matplotlib.colors as mcolors\n",
        "from matplotlib.pyplot import specgram \n",
        "import pandas as pd \n",
        "import IPython.display as ipd\n",
        "import matplotlib.style\n",
        "import matplotlib as mpl\n",
        "mpl.style.use('ggplot')\n",
        "import plotly.graph_objects as go\n",
        "couleurs = mcolors.CSS4_COLORS\n",
        "Tcolors = list(couleurs.keys())\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "from collections import deque\n",
        "import os\n",
        "from pysndfile import sndio\n",
        "import soundfile as sf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0XsO8LnVnmQ5",
        "colab_type": "text"
      },
      "source": [
        "### Utils ###"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KRkULRylne4G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n",
        "def path2info(path):\n",
        "    \"\"\"\n",
        "    path2info: parses paths in the TIDIGIT format and extracts information\n",
        "               about the speaker and the utterance\n",
        "\n",
        "    Example:\n",
        "    path2info('tidigits/disc_4.1.1/tidigits/train/man/ae/z9z6531a.wav')\n",
        "    \"\"\"\n",
        "    rest, filename = os.path.split(path)\n",
        "    rest, speakerID = os.path.split(rest)\n",
        "    rest, gender = os.path.split(rest)\n",
        "    digits = filename[:-5]\n",
        "    repetition = filename[-5]\n",
        "    return gender, speakerID, digits, repetition\n",
        "\n",
        "\n",
        "def loadAudio(filename):\n",
        "    \"\"\"\n",
        "    loadAudio: loads audio data from file using pysndfile\n",
        "\n",
        "    Note that, by default pysndfile converts the samples into floating point\n",
        "    numbers and rescales them in the range [-1, 1]. This is avoided by specifying\n",
        "    the option dtype=np.int16 which keeps both the original data type and range\n",
        "    of values.\n",
        "    \"\"\"\n",
        "    sndobj = sndio.read(filename, dtype=np.int16)\n",
        "    samplingrate = sndobj[1]\n",
        "    samples = np.array(sndobj[0])\n",
        "    return sf.read(filename, dtype='int16')\n",
        "\n",
        "\n",
        "def frames2trans(sequence, outfilename=None, timestep=0.01):\n",
        "    \"\"\"\n",
        "    Outputs a standard transcription given a frame-by-frame\n",
        "    list of strings.\n",
        "\n",
        "    Example (using functions from Lab 1 and Lab 2):\n",
        "    phones = ['sil', 'sil', 'sil', 'ow', 'ow', 'ow', 'ow', 'ow', 'sil', 'sil']\n",
        "    trans = frames2trans(phones, 'oa.lab')\n",
        "\n",
        "    Then you can use, for example wavesurfer to open the wav file and the transcription\n",
        "    \"\"\"\n",
        "    sym = sequence[0]\n",
        "    start = 0\n",
        "    end = 0\n",
        "    trans = ''\n",
        "    for t in range(len(sequence)):\n",
        "        if sequence[t] != sym:\n",
        "            trans = trans + str(start) + ' ' + str(end) + ' ' + sym + '\\n'\n",
        "            sym = sequence[t]\n",
        "            start = end\n",
        "        end = end + timestep\n",
        "    trans = trans + str(start) + ' ' + str(end) + ' ' + sym + '\\n'\n",
        "    if outfilename != None:\n",
        "        with open(outfilename, 'w') as f:\n",
        "            f.write(trans)\n",
        "    return trans\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "phpuI7H3CrfF",
        "colab_type": "text"
      },
      "source": [
        "### From Lab 1 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6NQI68o9CuJm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from scipy import fftpack\n",
        "\n",
        "def enframe(samples, Wlen, Wshift):\n",
        "    \"\"\"\n",
        "    Slices the input samples into overlapping windows.\n",
        "\n",
        "    Args:\n",
        "        Wlen : window length in samples.\n",
        "        Wshift: shift of consecutive windows in samples\n",
        "    Returns:\n",
        "        numpy array [N x Wlen], where N is the number of windows that fit\n",
        "        in the input signal\n",
        "        and \n",
        "        numpy array [Wlen x N ], where N is the number of windows that fit\n",
        "        in the input signal\n",
        "     tests :\n",
        "     frames, _ = enframe(np.array([1,2,3,4,5,6,7,8]),4, 1)\n",
        "     print(frames)\n",
        "    \"\"\"\n",
        "\n",
        "    L = len(samples)\n",
        "    K = math.floor((L - Wlen) / Wshift ) + 1\n",
        "    \n",
        "    all_frames = []\n",
        "    for i in range(K) :        \n",
        "        frame = samples[i*Wshift : i*Wshift + Wlen  ]\n",
        "        all_frames.append(frame)\n",
        "\n",
        "    return np.array(all_frames), np.array(all_frames).T\n",
        "\n",
        "def preemp(frames, coef=0.97):\n",
        "    \"\"\"\n",
        "    Pre-emphasis filter.\n",
        "\n",
        "    Args:\n",
        "        y: array of speech frames [N x M] where N is the number of frames and\n",
        "               M the samples per frame\n",
        "        coef: preemhasis factor (defaults to the value specified in the exercise)\n",
        "\n",
        "    Output:\n",
        "        output: array of pre-emphasised speech samples\n",
        "    Note (you can use the function lfilter from scipy.signal)\n",
        "    \"\"\"\n",
        "    def pre_emphasis(y, coef) :\n",
        "        \n",
        "        b = np.asarray([1.0, -coef])\n",
        "        a = np.asarray([1.0])\n",
        "        y_filtered = scipy.signal.lfilter(b, a, y)\n",
        "        return y_filtered\n",
        "        \n",
        "    row, _ = np.shape(frames)\n",
        "    \n",
        "    all_frames = []\n",
        "    for i in range(row) :\n",
        "        \n",
        "        temp_frame = frames[i, :]\n",
        "        emp_tmp = pre_emphasis(temp_frame, coef)\n",
        "        all_frames.append(emp_tmp.tolist())\n",
        "            \n",
        "    return np.array(all_frames)\n",
        "\n",
        "def windowing(input):\n",
        "    \"\"\"\n",
        "    Applies hamming window to the input frames.\n",
        "\n",
        "    Args:\n",
        "        input: array of speech samples [N x M] where N is the number of frames and\n",
        "               M the samples per frame\n",
        "    Output:\n",
        "        array of windoed speech samples [N x M]\n",
        "    Note (you can use the function hamming from scipy.signal, include the sym=0 option\n",
        "    if you want to get the same results as in the example)\n",
        "\n",
        "    # plotting hamming window\n",
        "    w = signal.hamming(100, sym=0)\n",
        "    plt.plot(np.linspace(-1, 1, 100), w)\n",
        "    # test on frames\n",
        "    print(example_frames)\n",
        "    xw = windowing(example_frames)\n",
        "    print(xw)\n",
        "    \"\"\"\n",
        "    w = signal.hamming(input.shape[1], sym=0)\n",
        "    res = np.ones(input.shape)*input\n",
        "    res *= w\n",
        "    return res, w\n",
        "\n",
        "def powerSpectrum(data, nfft=512):\n",
        "    from scipy.fftpack import fft\n",
        "    \"\"\"\n",
        "    Calculates the power spectrum of the input signal, that is the square of the modulus of the FFT\n",
        "\n",
        "    Args:\n",
        "        input: array of speech samples [N x M] where N is the number of frames and\n",
        "               M the samples per frame\n",
        "        nfft: length of the FFT\n",
        "    Output:\n",
        "        array of power spectra [N x nfft]\n",
        "    Note: you can use the function fft from scipy.fftpack\n",
        "    ps = powerSpectrum(xw, 512)\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.pcolormesh(ps)\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.pcolormesh(spec)\n",
        "    plt.show()\n",
        "    \"\"\"\n",
        "    fft_x = fft(data, nfft)\n",
        "\n",
        "    return np.square(np.abs(fft_x))\n",
        "\n",
        "def trfbank(fs, nfft, lowfreq=133.33, linsc=200/3., logsc=1.0711703, nlinfilt=13, nlogfilt=27, equalareas=False):\n",
        "    \"\"\"Compute triangular filterbank for MFCC computation.\n",
        "\n",
        "    Inputs:\n",
        "    fs:         sampling frequency (rate)\n",
        "    nfft:       length of the fft\n",
        "    lowfreq:    frequency of the lowest filter\n",
        "    linsc:      scale for the linear filters\n",
        "    logsc:      scale for the logaritmic filters\n",
        "    nlinfilt:   number of linear filters\n",
        "    nlogfilt:   number of log filters\n",
        "\n",
        "    Outputs:\n",
        "    res:  array with shape [N, nfft], with filter amplitudes for each column.\n",
        "            (N=nlinfilt+nlogfilt)\n",
        "    From scikits.talkbox\"\"\"\n",
        "    # Total number of filters\n",
        "    nfilt = nlinfilt + nlogfilt\n",
        "\n",
        "    #------------------------\n",
        "    # Compute the filter bank\n",
        "    #------------------------\n",
        "    # Compute start/middle/end points of the triangular filters in spectral\n",
        "    # domain\n",
        "    freqs = np.zeros(nfilt+2)\n",
        "    freqs[:nlinfilt] = lowfreq + np.arange(nlinfilt) * linsc\n",
        "    freqs[nlinfilt:] = freqs[nlinfilt-1] * logsc ** np.arange(1, nlogfilt + 3)\n",
        "    if equalareas:\n",
        "        heights = np.ones(nfilt)\n",
        "    else:\n",
        "        heights = 2./(freqs[2:] - freqs[0:-2])\n",
        "\n",
        "    # Compute filterbank coeff (in fft domain, in bins)\n",
        "    fbank = np.zeros((nfilt, nfft))\n",
        "    # FFT bins (in Hz)\n",
        "    nfreqs = np.arange(nfft) / (1. * nfft) * fs\n",
        "    for i in range(nfilt):\n",
        "        low = freqs[i]\n",
        "        cen = freqs[i+1]\n",
        "        hi = freqs[i+2]\n",
        "\n",
        "        lid = np.arange(np.floor(low * nfft / fs) + 1,\n",
        "                        np.floor(cen * nfft / fs) + 1, dtype=np.int)\n",
        "        lslope = heights[i] / (cen - low)\n",
        "        rid = np.arange(np.floor(cen * nfft / fs) + 1,\n",
        "                        np.floor(hi * nfft / fs) + 1, dtype=np.int)\n",
        "        rslope = heights[i] / (hi - cen)\n",
        "        fbank[i][lid] = lslope * (nfreqs[lid] - low)\n",
        "        fbank[i][rid] = rslope * (hi - nfreqs[rid])\n",
        "\n",
        "    return fbank\n",
        "\n",
        "def logMelSpectrum(input, samplingrate):\n",
        "    \"\"\"\n",
        "    Calculates the log output of a Mel filterbank when the input is the power spectrum\n",
        "\n",
        "    Args:\n",
        "        input: array of power spectrum coefficients [N x nfft] where N is the number of frames and\n",
        "               nfft the length of each spectrum\n",
        "        samplingrate: sampling rate of the original signal (used to calculate the filterbank shapes)\n",
        "    Output:\n",
        "        array of Mel filterbank log outputs [N x nmelfilters] where nmelfilters is the number\n",
        "        of filters in the filterbank\n",
        "    Note: use the trfbank function provided in lab1_tools.py to calculate the filterbank shapes and\n",
        "          nmelfilters\n",
        "\n",
        "    lms = logMelSpectrum(ps, sampling_rate)\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.pcolormesh(lms)\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.pcolormesh(mspec)\n",
        "    plt.show()\n",
        "    \"\"\"\n",
        "    mfcc = trfbank(samplingrate, input.shape[1])\n",
        "    res = np.log(np.matmul(input, mfcc.T))\n",
        "    return res, mfcc\n",
        "\n",
        "def lifter(mfcc, lifter=22):\n",
        "    \"\"\"\n",
        "    Applies liftering to improve the relative range of MFCC coefficients.\n",
        "\n",
        "       mfcc: NxM matrix where N is the number of frames and M the number of MFCC coefficients\n",
        "       lifter: lifering coefficient\n",
        "\n",
        "    Returns:\n",
        "       NxM array with lifeterd coefficients\n",
        "    \"\"\"\n",
        "    nframes, nceps = mfcc.shape\n",
        "    cepwin = 1.0 + lifter/2.0 * np.sin(np.pi * np.arange(nceps) / lifter)\n",
        "    return np.multiply(mfcc, np.tile(cepwin, nframes).reshape((nframes,nceps)))\n",
        "\n",
        "def cepstrum(input, nceps = 13):        \n",
        "    return fftpack.realtransforms.dct(input)[:,:nceps]\n",
        "\n",
        "def mspec(samples, winlen = 400, winshift = 200, preempcoeff=0.97, nfft=512, samplingrate=20000):\n",
        "    \"\"\"Computes Mel Filterbank features.\n",
        "\n",
        "    Args:\n",
        "        samples: array of speech samples with shape (N,)\n",
        "        winlen: lenght of the analysis window\n",
        "        winshift: number of samples to shift the analysis window at every time step\n",
        "        preempcoeff: pre-emphasis coefficient\n",
        "        nfft: length of the Fast Fourier Transform (power of 2, >= winlen)\n",
        "        samplingrate: sampling rate of the original signal\n",
        "\n",
        "    Returns:\n",
        "        N x nfilters array with mel filterbank features (see trfbank for nfilters)\n",
        "    \"\"\"\n",
        "    frames, _ = enframe(samples, winlen, winshift)\n",
        "    preemph = preemp(frames, preempcoeff)\n",
        "    windowed,_ = windowing(preemph)\n",
        "    spec = powerSpectrum(windowed, nfft)\n",
        "    MelSpectrum, filterBanks = logMelSpectrum(spec, samplingrate)\n",
        "\n",
        "    return MelSpectrum\n",
        "\n",
        "def mfcc_func(samples, winlen = 400, winshift = 200, preempcoeff=0.97, nfft=512, nceps=13, samplingrate=20000, liftercoeff=22):\n",
        "    \"\"\"Computes Mel Frequency Cepstrum Coefficients.\n",
        "    Args:\n",
        "        samples: array of speech samples with shape (N,)\n",
        "        winlen: lenght of the analysis window\n",
        "        winshift: number of samples to shift the analysis window at every time step\n",
        "        preempcoeff: pre-emphasis coefficient\n",
        "        nfft: length of the Fast Fourier Transform (power of 2, >= winlen)\n",
        "        nceps: number of cepstrum coefficients to compute\n",
        "        samplingrate: sampling rate of the original signal\n",
        "        liftercoeff: liftering coefficient used to equalise scale of MFCCs\n",
        "\n",
        "    Returns:\n",
        "        N x nceps array with lifetered MFCC coefficients\n",
        "    \"\"\"\n",
        "    mspecs = mspec(samples, winlen, winshift, preempcoeff, nfft, samplingrate)\n",
        "    ceps = cepstrum(mspecs, nceps)\n",
        "\n",
        "    return lifter(ceps, liftercoeff)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DQPqS2SV9ex0",
        "colab_type": "text"
      },
      "source": [
        "### From Lab 2 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GREqCq579kar",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def concatTwoHMMs(hmm1, hmm2):\n",
        "    \n",
        "    A_state_prob = hmm1['startprob']\n",
        "    A_transition = hmm1['transmat']\n",
        "    B_state_prob = hmm2['startprob']\n",
        "    B_transition = hmm2['transmat']\n",
        "    \n",
        "    new_length = len(A_state_prob) + len(B_state_prob) - 1\n",
        "    new_state_prob_A = np.ones(new_length)\n",
        "    new_state_prob_B = np.ones(new_length)\n",
        "    \n",
        "    # Create the new state probability\n",
        "    new_state_prob_B[len(A_state_prob)-1:] = B_state_prob\n",
        "    new_state_prob_A[:-(len(B_state_prob) - 1)] = A_state_prob\n",
        "    new_state_prob_A[len(A_state_prob):] = A_state_prob[-1]\n",
        "\n",
        "    new_state_prob_AB = new_state_prob_A * new_state_prob_B\n",
        "    \n",
        "    # Create the new transition matrix A \n",
        "    A_transition_concat = np.zeros((new_length,new_length))\n",
        "    row_A, col_A = np.shape(A_transition)\n",
        "    A_transition_concat[:row_A-1,:col_A] = A_transition[:row_A-1,:]\n",
        "    \n",
        "    form = np.shape(A_transition_concat[:row_A-1,col_A:])\n",
        "    if 1 in form : \n",
        "        A_transition_concat[:row_A-1,col_A:] = np.reshape(A_transition[:row_A-1,-1], form) \n",
        "    else :        \n",
        "        temp_vec = np.repeat(A_transition[:row_A-1,-1], len(B_state_prob) - 1)\n",
        "        temp_vec_reshape = np.reshape( temp_vec,(len(A_state_prob) - 1, len(B_state_prob) - 1))        \n",
        "        A_transition_concat[:row_A-1,col_A:] = np.reshape( temp_vec,(len(A_state_prob) - 1, len(B_state_prob) - 1))        \n",
        "        \n",
        "    A_transition_concat[:row_A-1,] = A_transition_concat[:row_A-1,] * new_state_prob_B\n",
        "    A_transition_concat[row_A-1:,col_A-1:] = B_transition\n",
        "    \n",
        "    A_means = hmm1['means']\n",
        "    B_means = hmm2['means']\n",
        "    \n",
        "    A_cov = hmm1['covars']\n",
        "    B_cov = hmm2['covars']    \n",
        "    AB_means = np.concatenate((A_means, B_means))\n",
        "    AB_cov = np.concatenate((A_cov, B_cov))    \n",
        "    \n",
        "    combinedHmms = {'name': hmm1['name'] + '_' +  hmm2['name'], \n",
        "                    'startprob': new_state_prob_AB, \n",
        "                    'transmat': A_transition_concat, \n",
        "                    'means':AB_means, \n",
        "                    'covars':AB_cov}\n",
        "        \n",
        "    return combinedHmms\n",
        "\n",
        "def concatHMMs(hmmmodels, namelist):\n",
        "    concat = hmmmodels[namelist[0]]\n",
        "    for idx in range(1,len(namelist)):\n",
        "        concat = concatTwoHMMs(concat, hmmmodels[namelist[idx]])\n",
        "    return concat\n",
        "\n",
        "def log_multivariate_normal_density_diag(X, means, covars):\n",
        "    \"\"\"Compute Gaussian log-density at X for a diagonal model\n",
        "\n",
        "    Args:\n",
        "        X: array like, shape (n_observations, n_features)\n",
        "        means: array like, shape (n_components, n_features)\n",
        "        covars: array like, shape (n_components, n_features)\n",
        "\n",
        "    Output:\n",
        "        lpr: array like, shape (n_observations, n_components)\n",
        "    From scikit-learn/sklearn/mixture/gmm.py\n",
        "    \"\"\"\n",
        "    n_samples, n_dim = X.shape\n",
        "    lpr = -0.5 * (n_dim * np.log(2 * np.pi) + np.sum(np.log(covars), 1)\n",
        "                  + np.sum((means ** 2) / covars, 1)\n",
        "                  - 2 * np.dot(X, (means / covars).T)\n",
        "                  + np.dot(X ** 2, (1.0 / covars).T))\n",
        "    return lpr\n",
        "    \n",
        "def viterbi(log_emlik, log_startprob, log_transmat, forceFinalState=True):\n",
        "    \"\"\"Viterbi path.\n",
        "\n",
        "    Args:\n",
        "        log_emlik: NxM array of emission log likelihoods, N frames, M states\n",
        "        log_startprob: log probability to start in state i\n",
        "        log_transmat: transition log probability from state i to j\n",
        "        forceFinalState: if True, start backtracking from the final state in\n",
        "                  the model, instead of the best state at the last time step\n",
        "\n",
        "    Output:\n",
        "        viterbi_loglik: log likelihood of the best path\n",
        "        viterbi_path: best path\n",
        "    \"\"\"\n",
        "    N, M = log_emlik.shape\n",
        "    \n",
        "    # initialization\n",
        "    viterbi_loglik = np.zeros((N, M))\n",
        "    memo_path = np.zeros((N, M), dtype=int)\n",
        "    viterbi_loglik[0,:] = log_startprob[:-1] + log_emlik[0, :]\n",
        "    \n",
        "    # induction \n",
        "    for i in range(1, N):\n",
        "        viterbi_loglik[i, :] = np.max(viterbi_loglik[i-1, :] + log_transmat[:-1, :-1].T, 1) + log_emlik[i, :]\n",
        "        memo_path[i, :] = np.argmax(viterbi_loglik[i-1, :] + log_transmat[:-1, :-1].T, 1)\n",
        "\n",
        "    # backtracking\n",
        "    if forceFinalState:\n",
        "        lastState = M-1\n",
        "    else:\n",
        "        lastState = np.argmax(memo_path[-1, :])\n",
        "\n",
        "    viterbi_path = deque()\n",
        "    viterbi_path.append(lastState)\n",
        "\n",
        "    for i in range(N-2, -1, -1):\n",
        "        viterbi_path.appendleft(memo_path[i+1, viterbi_path[0]])    \n",
        "\n",
        "    return max(viterbi_loglik[-1, :]), viterbi_path"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Bi5x_pJng1R",
        "colab_type": "text"
      },
      "source": [
        "### Data handling functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luEBSsh5D8is",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        },
        "outputId": "004c6afe-51e9-4369-ed8f-5b8302c2ee63"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BS4RYpxbngEa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_data():\n",
        "    # data = np.load('lab2_data.npz', allow_pickle=True)['data'][0]\n",
        "    return data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ejaFZPZpQUc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 273
        },
        "outputId": "37ce2446-d03d-43e6-ac84-cf31783a1c08"
      },
      "source": [
        "# data = read_data()\n",
        "# print(data)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'filename': '/home/giampi/corpora/tidigits/disc_4.2.1/tidigits/test/man/bm/oa.wav', 'samplingrate': 20000, 'gender': 'man', 'speaker': 'bm', 'digit': 'o', 'repetition': 'a', 'samples': array([-2, -1, -3, ..., 10,  8,  5], dtype=int16), 'lmfcc': array([[  64.7657709 ,  -65.84854216,   96.90766069, ...,  -19.82632821,\n",
            "          11.63249931,  159.50790488],\n",
            "       [  57.72402791,  -79.93732698,  105.80893583, ...,  -11.84501965,\n",
            "         109.29701349,  138.79176124],\n",
            "       [  62.16944867, -121.81014369,   61.89899215, ...,   13.28252915,\n",
            "        -127.7140298 , -128.75563442],\n",
            "       ...,\n",
            "       [ 190.29491187,   97.34488369,  175.61357289, ...,  -62.63035172,\n",
            "         138.96640354,   82.95734165],\n",
            "       [ 201.37123403,   80.09496653,   50.98133557, ...,  219.16711476,\n",
            "          94.41131525, -100.10312668],\n",
            "       [ 194.17067167,   70.85205455,   40.456538  , ...,   -3.81053312,\n",
            "         143.83615904,  -42.56212766]])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9OLJP7wWnsIE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QmW9tH6MpL-k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UTP3nn2lpM2G",
        "colab_type": "text"
      },
      "source": [
        "# Code Implementation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dGfiuNFpnvPO",
        "colab_type": "text"
      },
      "source": [
        "## 4. Data preparation for DNN training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ks9Wcmmfn8Pg",
        "colab_type": "text"
      },
      "source": [
        "### 4.1. Target class definition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrW-y0ymoCSL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tr(filename) :\n",
        "    hmms = np.load(filename)['phoneHMMs'].item()\n",
        "    phones = sorted(hmms.keys())\n",
        "    nstates = {phone: hmms[phone]['means'].shape[0] for phone in phones}\n",
        "    states = [ph + '_' + str(id) for ph in phones for id in range(nstates[ph])]\n",
        "    return states"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cNGiOUQF23xF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "states = tr('lab2_models_all.npz')\n",
        "np.save(states, \"stateList.npz\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z-81xj59oGda",
        "colab_type": "text"
      },
      "source": [
        "### 4.2. Forced Alignment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zizA5tt-oKSp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def words2phones(wordList, pronDict, addSilence=True, addShortPause=True):\n",
        "    \"\"\" word2phones: converts word level to phone level transcription adding silence\n",
        "\n",
        "    Args:\n",
        "       wordList: list of word symbols\n",
        "       pronDict: pronunciation dictionary. The keys correspond to words in wordList\n",
        "       addSilence: if True, add initial and final silence\n",
        "       addShortPause: if True, add short pause model \"sp\" at end of each word\n",
        "    Output:\n",
        "       list of phone symbols\n",
        "    \"\"\"\n",
        "    sil = []\n",
        "    sp = []\n",
        "    if addSilence:\n",
        "        sil = ['sil']    \n",
        "    if addShortPause:\n",
        "        sp = ['sp']\n",
        "\n",
        "    res = sil\n",
        "    for w in wordList: \n",
        "        res += pronDict[w] + sp\n",
        "    res += sil \n",
        "    return res"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IUWzNKeP93Jh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def forcedAlignment(lmfcc, phoneHMMs, phoneTrans):\n",
        "    \"\"\" forcedAlignmen: aligns a phonetic transcription at the state level\n",
        "\n",
        "    Args:\n",
        "       lmfcc: NxD array of MFCC feature vectors (N vectors of dimension D)\n",
        "              computed the same way as for the training of phoneHMMs\n",
        "       phoneHMMs: set of phonetic Gaussian HMM models\n",
        "       phoneTrans: list of phonetic symbols to be aligned including initial and\n",
        "                   final silence\n",
        "\n",
        "    Returns:\n",
        "       list of strings in the form phoneme_index specifying, for each time step\n",
        "       the state from phoneHMMs corresponding to the viterbi path.\n",
        "    \"\"\"\n",
        "    utteranceHMM = concatHMMs(phoneHMMs, phoneTrans)\n",
        "    \n",
        "    phones = sorted(phoneHMMs.keys())\n",
        "    nstates = {phone: phoneHMMs[phone]['means'].shape[0] for phone in phones}\n",
        "    stateTrans = [phone + '_' + str(stateid) for phone in phoneTrans for stateid in range(nstates[phone])]\n",
        "\n",
        "    log_emlik = log_multivariate_normal_density_diag(lmfcc, utteranceHMM['means'], utteranceHMM['covars'])\n",
        "    log_startprob = np.log(utteranceHMM['startprob'])\n",
        "    log_transmat = np.log(utteranceHMM['transmat'])\n",
        "\n",
        "    _, viterbi_path = viterbi(log_emlik, log_startprob, log_transmat)\n",
        "\n",
        "    res = [stateTrans[i] for i in viterbi_path]\n",
        "    return res"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dE-3XAczoKqV",
        "colab_type": "text"
      },
      "source": [
        "### 4.3. Features Extraction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6H5EDtZ6oYGL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P1zuGeh0oYfz",
        "colab_type": "text"
      },
      "source": [
        "### 4.4. Training and Validation sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTxjtTitoeDe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j6XjdBORoeUB",
        "colab_type": "text"
      },
      "source": [
        "### 4.5. Acoustic Context (Dynamic Features)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JL3li9GOoj30",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H6E2f2pbokW3",
        "colab_type": "text"
      },
      "source": [
        "### 4.6. Feature Standardisation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "faRqUiTCoppm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lX_6cgOHnsgW",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "94tISZimos4w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zI8aUXUSotrs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IO5AMixNouP6",
        "colab_type": "text"
      },
      "source": [
        "## 5. Phoneme Recognition with Deep Neural Networks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8cEIT1wCzlT9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model building\n",
        "batch_size = 256\n",
        "max_epochs  = 10\n",
        "depth = 2\n",
        "units = 256\n",
        "classes = 12 #todo change depending on output \n",
        "\n",
        "def build_model(input_shape, units, depth, classes):\n",
        "    model = keras.Sequential()\n",
        "    model.add(keras.layers.Dense(units , input_shape=input_shape, activation='relu'))\n",
        "    for _ in range(depth):\n",
        "        model.add(model.add(keras.layers.Dense(units , input_shape=tuple(input_shape[:-1])+(units,), activation='relu')))\n",
        "    model.add(keras.layers.Dense(classes , input_shape=tuple(input_shape[:-1])+(8,), activation='softmax'))    \n",
        "    return model\n",
        "\n",
        "model = build_model(input_shape, units, depth, classes ) # todo add input shape "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36hkN5kQAaK1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# compiling + training \n",
        "model.compile(\n",
        "    optimizer='adam', \n",
        "    loss='sparse_categorical_crossentropy',\n",
        "    metrics =['sparse_categorical_accuracy',\n",
        "              'sparse_categorical_crossentropy']              \n",
        "            )\n",
        "\n",
        "# todo get training + validation data \n",
        "model.fit(x_train, y_train, batch_size=batch_size, epochs=max_epochs, validation_data=(x_valid, y_valid))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZuNmMr1o3Su",
        "colab_type": "text"
      },
      "source": [
        "### 5.1. Detailed Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bpZ91JYyo921",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bDjvSEzyo-S1",
        "colab_type": "text"
      },
      "source": [
        "### 5.2. Possible questions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ONO7YkP8ph4j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aVqIt1iypiTL",
        "colab_type": "text"
      },
      "source": [
        "# Code Demonstation and Answers to questions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYgzojv67X6D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# prondict \n",
        "prondict = {} \n",
        "prondict['o'] = ['ow']\n",
        "prondict['z'] = ['z', 'iy', 'r', 'ow']\n",
        "prondict['1'] = ['w', 'ah', 'n']\n",
        "prondict['2'] = ['t', 'uw']\n",
        "prondict['3'] = ['th', 'r', 'iy']\n",
        "prondict['4'] = ['f', 'ao', 'r']\n",
        "prondict['5'] = ['f', 'ay', 'v']\n",
        "prondict['6'] = ['s', 'ih', 'k', 's']\n",
        "prondict['7'] = ['s', 'eh', 'v', 'ah', 'n']\n",
        "prondict['8'] = ['ey', 't']\n",
        "prondict['9'] = ['n', 'ay', 'n']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-neu1Pi16v7l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        },
        "outputId": "e0a3ee8a-0847-4595-e7dd-62b20fa1bab1"
      },
      "source": [
        "# Show HMMs \n",
        "phoneHMMs = np.load('lab2_models_all.npz', allow_pickle=True)['phoneHMMs'].item()\n",
        "# print(phoneHMMs)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'name': 'sp', 'startprob': array([0.1216644, 0.8783357]), 'transmat': array([[0.8656793, 0.1343206],\n",
            "       [0.       , 1.       ]]), 'means': array([[  38.44177  , -129.2183   ,   54.57983  ,   15.29446  ,\n",
            "          35.78878  ,   -6.647676 ,  -11.7913   ,  -25.44149  ,\n",
            "         -20.31962  ,  -27.77863  ,   -5.970622 ,    0.9313969,\n",
            "          22.0168   ]]), 'covars': array([[ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251]])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzMIEa_Ept0H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6e2491ed-bce5-402c-f148-853913ea34f2"
      },
      "source": [
        "# show example for lab3\n",
        "example = np.load('lab3_example.npz', allow_pickle=True)\n",
        "example = example['example'].item()\n",
        "# print(example)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'filename': 'tidigits/disc_4.1.1/tidigits/train/man/nw/z43a.wav', 'samples': array([ 1,  1,  1, ..., -3, -2, -2], dtype=int16), 'gender': 'man', 'speaker': 'nw', 'digits': 'z43', 'repetition': 'a', 'lmfcc': array([[ -22.91392389, -189.9582419 ,    7.40105799, ..., -112.7883752 ,\n",
            "          41.12045179,  -12.04285405],\n",
            "       [  -6.81710936, -187.14843532,  -24.39690243, ...,  -94.4723144 ,\n",
            "        -198.74795293, -127.98350389],\n",
            "       [ -11.92109946, -204.41816753,  -53.65419198, ...,  -95.38523624,\n",
            "         -14.04657624,  -66.81715806],\n",
            "       ...,\n",
            "       [ 154.81267871,   41.12510125,  -11.03644377, ..., -251.4617779 ,\n",
            "         -86.72418937,  159.01415406],\n",
            "       [ 163.1202934 ,   15.30942337,  -15.99373122, ...,  -38.20996944,\n",
            "         -78.60225893,  233.63308676],\n",
            "       [ 143.95962157,    4.19763652,  -38.44629773, ...,  -57.65568221,\n",
            "          23.2052515 ,  225.60552906]]), 'wordTrans': ['z', '4', '3'], 'phoneTrans': ['sil', 'z', 'iy', 'r', 'ow', 'sp', 'f', 'ao', 'r', 'sp', 'th', 'r', 'iy', 'sp', 'sil'], 'utteranceHMM': {'startprob': array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "       0., 0., 0., 0., 0., 0.]), 'transmat': array([[0.8833106 , 0.1166894 , 0.        , ..., 0.        , 0.        ,\n",
            "        0.        ],\n",
            "       [0.        , 0.9164465 , 0.08355348, ..., 0.        , 0.        ,\n",
            "        0.        ],\n",
            "       [0.        , 0.        , 0.802877  , ..., 0.        , 0.        ,\n",
            "        0.        ],\n",
            "       ...,\n",
            "       [0.        , 0.        , 0.        , ..., 0.9164465 , 0.08355348,\n",
            "        0.        ],\n",
            "       [0.        , 0.        , 0.        , ..., 0.        , 0.802877  ,\n",
            "        0.197123  ],\n",
            "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
            "        1.        ]]), 'means': array([[ 1.929856e+02, -1.658209e+01, -2.290160e+01,  3.132330e+01,\n",
            "         4.658450e+01, -1.151792e+00,  1.339505e+01,  6.989146e+01,\n",
            "         6.058731e+01,  1.809310e+01,  1.219584e+01, -2.165135e+01,\n",
            "         3.100604e+01],\n",
            "       [ 3.844177e+01, -1.292183e+02,  5.457983e+01,  1.529446e+01,\n",
            "         3.578878e+01, -6.647676e+00, -1.179130e+01, -2.544149e+01,\n",
            "        -2.031962e+01, -2.777863e+01, -5.970622e+00,  9.313969e-01,\n",
            "         2.201680e+01],\n",
            "       [ 1.456066e+02, -1.024690e+02,  1.549553e+01,  4.118528e+01,\n",
            "         3.102415e+01,  4.470313e+00, -2.037535e-01,  4.449736e+00,\n",
            "        -2.242693e+00, -1.644482e+01,  1.019933e+01, -1.426286e+01,\n",
            "         2.725467e+01],\n",
            "       [ 4.297425e+02, -7.732635e+00,  1.407858e+02,  1.973845e+02,\n",
            "         1.959355e+02,  9.630707e+01,  1.947806e+01,  7.610709e+01,\n",
            "         6.045083e+01,  2.062691e+01, -1.139942e+01, -3.786816e+01,\n",
            "         7.585356e+01],\n",
            "       [ 5.429565e+02, -1.915233e+02,  2.532834e+02,  4.858263e+01,\n",
            "         2.579908e+02,  1.568936e+01,  1.443086e+02,  8.392543e+01,\n",
            "         7.483309e+01, -1.047346e+01,  5.813360e+01,  2.944131e+01,\n",
            "         4.556281e+01],\n",
            "       [ 6.947541e+02, -1.871409e+02,  2.508336e+02,  1.882890e+02,\n",
            "         1.563175e+02,  2.364693e+02, -6.773640e+01,  7.166970e+01,\n",
            "         2.378812e+01,  2.560293e+01, -3.839036e+01, -1.451380e+02,\n",
            "         7.080281e+01],\n",
            "       [ 8.234090e+02,  5.708237e+01, -4.115715e+01,  6.295295e+02,\n",
            "        -1.614770e+01,  3.887735e+01, -4.235817e+01,  8.912155e+01,\n",
            "        -1.088613e+02, -1.559888e+02,  1.537681e+01, -1.297389e+02,\n",
            "         9.595664e+00],\n",
            "       [ 6.937938e+02,  5.366372e+01,  8.445095e+01,  5.915940e+02,\n",
            "        -6.038222e+01,  9.092448e+01,  6.073024e+01,  3.655795e+01,\n",
            "        -1.321739e+02, -2.305365e+01, -6.006158e+01, -1.212376e+02,\n",
            "         4.715969e+01],\n",
            "       [ 5.357452e+02,  3.995380e+01,  4.675761e+01,  3.695593e+02,\n",
            "         7.749891e+00,  5.165934e+01,  6.529175e+01,  4.378563e+01,\n",
            "        -9.846682e+00, -7.514572e+00, -1.821162e+01, -6.183767e+01,\n",
            "         3.507554e+01],\n",
            "       [ 7.715482e+02,  3.561717e+02, -1.623403e+02,  3.816193e+01,\n",
            "         1.615007e+02,  2.639491e+02, -1.554496e+02, -2.059074e+02,\n",
            "         2.566075e+01, -1.588664e+02, -5.181502e+01, -3.735429e+01,\n",
            "        -2.496622e+01],\n",
            "       [ 8.012057e+02,  3.102308e+02, -2.091482e+02,  1.208269e+02,\n",
            "         1.572916e+02,  7.271638e+01, -3.849624e+02,  3.924157e+01,\n",
            "        -1.499091e+01, -1.197249e+02,  6.513354e+01, -4.055899e+01,\n",
            "         4.362951e+01],\n",
            "       [ 6.395913e+02,  1.907782e+02, -1.655498e+02,  1.498761e+02,\n",
            "         1.267901e+02,  4.037565e+01, -1.714349e+02,  5.141867e+01,\n",
            "         6.331414e+01, -7.492874e+01,  5.541277e+01, -2.773405e+00,\n",
            "         2.672367e+01],\n",
            "       [ 8.336846e+02,  2.257610e+02, -1.805328e+02,  5.326767e+01,\n",
            "        -1.491571e+02,  5.639893e+01, -2.138301e+02, -5.055334e+01,\n",
            "        -3.105238e+01,  1.430282e+02,  7.740034e+01, -5.422772e+01,\n",
            "         1.111027e+02],\n",
            "       [ 8.436029e+02,  2.941522e+02, -6.612854e+01,  4.672765e+01,\n",
            "        -2.901150e+02,  1.185553e+02,  2.453329e+01, -1.310316e+02,\n",
            "        -1.902861e+02, -1.270314e+02,  1.368871e+02, -2.220421e+00,\n",
            "         1.048858e+02],\n",
            "       [ 6.219299e+02,  2.806245e+02,  1.062467e+02,  8.319666e+01,\n",
            "        -2.102736e+02, -3.387125e+00,  9.862711e+01,  1.349704e+01,\n",
            "        -1.211751e+01, -7.869103e+01, -1.750259e+01, -8.942413e+01,\n",
            "         2.031466e+01],\n",
            "       [ 3.844177e+01, -1.292183e+02,  5.457983e+01,  1.529446e+01,\n",
            "         3.578878e+01, -6.647676e+00, -1.179130e+01, -2.544149e+01,\n",
            "        -2.031962e+01, -2.777863e+01, -5.970622e+00,  9.313969e-01,\n",
            "         2.201680e+01],\n",
            "       [ 3.756838e+02, -1.892536e+02, -9.660455e+01,  4.465488e+01,\n",
            "         2.823588e+01, -4.992766e-01,  4.529542e-02, -2.909808e+01,\n",
            "         1.526023e+01, -3.026565e+01,  2.106727e+01, -3.956148e+01,\n",
            "         2.572369e+01],\n",
            "       [ 5.441120e+02, -1.801021e+02, -1.472212e+02, -1.085416e+01,\n",
            "        -2.093026e+01, -1.813961e+01, -1.523612e+01, -5.690070e+01,\n",
            "        -1.829125e+01, -4.970216e+01,  9.754507e+00, -5.453812e+01,\n",
            "         1.968687e+01],\n",
            "       [ 7.118344e+02,  2.438120e+02, -5.496016e+01,  8.645602e+00,\n",
            "        -1.945088e+02,  3.470830e+01,  3.832467e+01, -7.212926e+00,\n",
            "        -4.773862e+01, -2.433857e+01,  1.006693e+02, -2.232076e+01,\n",
            "         7.655536e+01],\n",
            "       [ 6.730516e+02,  3.758534e+02,  2.124013e+02,  1.033149e+02,\n",
            "        -3.282890e+02, -7.770670e+01, -6.749087e+01,  3.669387e+00,\n",
            "        -1.746071e+01, -1.392374e+01, -6.204184e+00, -1.384771e+02,\n",
            "         4.344584e+01],\n",
            "       [ 7.522375e+02,  3.517338e+02,  1.757862e+02,  2.128041e+02,\n",
            "        -2.923769e+02, -1.964972e+02, -5.816175e+01, -8.520471e+01,\n",
            "         3.749348e+01, -1.497836e+02,  1.037072e+02, -1.114539e+02,\n",
            "        -9.573063e+01],\n",
            "       [ 7.841647e+02,  3.970841e+02, -3.927449e+01, -9.980502e+01,\n",
            "        -1.694217e+02,  4.199920e+01, -1.092114e+02,  2.025795e+01,\n",
            "        -3.102019e+01, -2.342697e+02,  2.609059e+01, -3.232597e+01,\n",
            "         8.487835e+01],\n",
            "       [ 7.715482e+02,  3.561717e+02, -1.623403e+02,  3.816193e+01,\n",
            "         1.615007e+02,  2.639491e+02, -1.554496e+02, -2.059074e+02,\n",
            "         2.566075e+01, -1.588664e+02, -5.181502e+01, -3.735429e+01,\n",
            "        -2.496622e+01],\n",
            "       [ 8.012057e+02,  3.102308e+02, -2.091482e+02,  1.208269e+02,\n",
            "         1.572916e+02,  7.271638e+01, -3.849624e+02,  3.924157e+01,\n",
            "        -1.499091e+01, -1.197249e+02,  6.513354e+01, -4.055899e+01,\n",
            "         4.362951e+01],\n",
            "       [ 6.395913e+02,  1.907782e+02, -1.655498e+02,  1.498761e+02,\n",
            "         1.267901e+02,  4.037565e+01, -1.714349e+02,  5.141867e+01,\n",
            "         6.331414e+01, -7.492874e+01,  5.541277e+01, -2.773405e+00,\n",
            "         2.672367e+01],\n",
            "       [ 3.844177e+01, -1.292183e+02,  5.457983e+01,  1.529446e+01,\n",
            "         3.578878e+01, -6.647676e+00, -1.179130e+01, -2.544149e+01,\n",
            "        -2.031962e+01, -2.777863e+01, -5.970622e+00,  9.313969e-01,\n",
            "         2.201680e+01],\n",
            "       [ 3.892380e+02, -1.635783e+02, -5.788988e-01,  1.795250e+01,\n",
            "         3.993346e+01, -4.482951e+01,  2.927515e+01, -2.233202e+01,\n",
            "         1.054525e+01, -4.802543e+01,  2.159875e+00, -2.678745e+01,\n",
            "         3.021246e+01],\n",
            "       [ 6.269700e+02, -1.405435e+02, -3.799828e+01,  1.549461e+02,\n",
            "        -3.079779e+01, -7.365781e+01, -2.215823e+01, -7.609867e+00,\n",
            "        -2.607820e+01, -2.560781e+01, -1.689876e+01, -4.329874e+01,\n",
            "         5.609790e+01],\n",
            "       [ 6.247905e+02,  1.693224e+02, -9.172640e+01,  2.268237e+02,\n",
            "         2.014226e+02,  1.101499e+02, -5.659327e+01, -1.626182e+01,\n",
            "        -2.076743e+01, -3.541732e+01,  9.763933e+00, -5.896887e+01,\n",
            "         2.102811e+01],\n",
            "       [ 7.715482e+02,  3.561717e+02, -1.623403e+02,  3.816193e+01,\n",
            "         1.615007e+02,  2.639491e+02, -1.554496e+02, -2.059074e+02,\n",
            "         2.566075e+01, -1.588664e+02, -5.181502e+01, -3.735429e+01,\n",
            "        -2.496622e+01],\n",
            "       [ 8.012057e+02,  3.102308e+02, -2.091482e+02,  1.208269e+02,\n",
            "         1.572916e+02,  7.271638e+01, -3.849624e+02,  3.924157e+01,\n",
            "        -1.499091e+01, -1.197249e+02,  6.513354e+01, -4.055899e+01,\n",
            "         4.362951e+01],\n",
            "       [ 6.395913e+02,  1.907782e+02, -1.655498e+02,  1.498761e+02,\n",
            "         1.267901e+02,  4.037565e+01, -1.714349e+02,  5.141867e+01,\n",
            "         6.331414e+01, -7.492874e+01,  5.541277e+01, -2.773405e+00,\n",
            "         2.672367e+01],\n",
            "       [ 8.234090e+02,  5.708237e+01, -4.115715e+01,  6.295295e+02,\n",
            "        -1.614770e+01,  3.887735e+01, -4.235817e+01,  8.912155e+01,\n",
            "        -1.088613e+02, -1.559888e+02,  1.537681e+01, -1.297389e+02,\n",
            "         9.595664e+00],\n",
            "       [ 6.937938e+02,  5.366372e+01,  8.445095e+01,  5.915940e+02,\n",
            "        -6.038222e+01,  9.092448e+01,  6.073024e+01,  3.655795e+01,\n",
            "        -1.321739e+02, -2.305365e+01, -6.006158e+01, -1.212376e+02,\n",
            "         4.715969e+01],\n",
            "       [ 5.357452e+02,  3.995380e+01,  4.675761e+01,  3.695593e+02,\n",
            "         7.749891e+00,  5.165934e+01,  6.529175e+01,  4.378563e+01,\n",
            "        -9.846682e+00, -7.514572e+00, -1.821162e+01, -6.183767e+01,\n",
            "         3.507554e+01],\n",
            "       [ 3.844177e+01, -1.292183e+02,  5.457983e+01,  1.529446e+01,\n",
            "         3.578878e+01, -6.647676e+00, -1.179130e+01, -2.544149e+01,\n",
            "        -2.031962e+01, -2.777863e+01, -5.970622e+00,  9.313969e-01,\n",
            "         2.201680e+01],\n",
            "       [ 1.929856e+02, -1.658209e+01, -2.290160e+01,  3.132330e+01,\n",
            "         4.658450e+01, -1.151792e+00,  1.339505e+01,  6.989146e+01,\n",
            "         6.058731e+01,  1.809310e+01,  1.219584e+01, -2.165135e+01,\n",
            "         3.100604e+01],\n",
            "       [ 3.844177e+01, -1.292183e+02,  5.457983e+01,  1.529446e+01,\n",
            "         3.578878e+01, -6.647676e+00, -1.179130e+01, -2.544149e+01,\n",
            "        -2.031962e+01, -2.777863e+01, -5.970622e+00,  9.313969e-01,\n",
            "         2.201680e+01],\n",
            "       [ 1.456066e+02, -1.024690e+02,  1.549553e+01,  4.118528e+01,\n",
            "         3.102415e+01,  4.470313e+00, -2.037535e-01,  4.449736e+00,\n",
            "        -2.242693e+00, -1.644482e+01,  1.019933e+01, -1.426286e+01,\n",
            "         2.725467e+01]]), 'covars': array([[ 8104.233,  5344.476, 10518.08 , 11515.44 , 11502.12 , 13264.83 ,\n",
            "        18537.86 , 17672.32 , 19276.32 , 19192.07 , 17322.32 , 14392.41 ,\n",
            "        12660.5  ],\n",
            "       [ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251],\n",
            "       [15756.68 ,  9520.698, 12481.49 , 10016.32 , 11632.07 , 12732.09 ,\n",
            "        15266.26 , 15419.72 , 15986.35 , 15035.07 , 13926.88 , 12311.71 ,\n",
            "        10783.33 ],\n",
            "       [39465.07 , 23308.37 , 14540.33 , 24652.89 , 24386.83 , 27933.59 ,\n",
            "        23967.96 , 22588.56 , 18902.   , 20411.23 , 25389.58 , 28818.95 ,\n",
            "        17564.75 ],\n",
            "       [10245.79 , 17912.87 , 14577.72 , 21151.28 , 22259.72 , 25174.8  ,\n",
            "        20851.05 , 16831.41 , 15644.79 , 15165.77 , 17887.35 , 15064.87 ,\n",
            "        11938.56 ],\n",
            "       [ 9269.596, 20293.8  , 10512.81 , 18118.48 , 24935.05 , 31669.71 ,\n",
            "        18841.8  , 22889.91 , 17041.15 , 16869.08 , 17255.78 , 20743.13 ,\n",
            "        13956.07 ],\n",
            "       [ 8118.612,  5110.032, 12845.01 , 15704.74 , 23408.29 , 17766.82 ,\n",
            "        21845.24 , 21889.9  , 64473.39 , 40553.73 , 22041.24 , 19536.16 ,\n",
            "        21640.13 ],\n",
            "       [16555.82 ,  6712.833, 25445.04 , 11565.54 , 29552.94 , 24277.09 ,\n",
            "        27636.95 , 25560.94 , 43225.9  , 40231.53 , 32650.18 , 21682.2  ,\n",
            "        20015.3  ],\n",
            "       [54722.85 , 18249.88 , 29889.41 , 18898.83 , 43132.42 , 31870.11 ,\n",
            "        60596.   , 36680.71 , 36877.51 , 41308.57 , 27701.82 , 22272.7  ,\n",
            "        20371.91 ],\n",
            "       [13747.17 ,  6064.09 , 11660.52 , 44131.81 , 21799.88 , 34285.92 ,\n",
            "        44999.81 , 41557.91 , 30851.62 , 26152.08 , 23482.44 , 18958.37 ,\n",
            "        17940.44 ],\n",
            "       [13024.28 ,  4491.389,  9199.829, 37471.12 , 18161.81 , 26462.47 ,\n",
            "        39266.54 , 45372.93 , 31142.87 , 33532.12 , 24271.95 , 16709.93 ,\n",
            "        15842.14 ],\n",
            "       [52920.68 , 12460.24 , 14539.01 , 70628.47 , 22720.44 , 28388.54 ,\n",
            "        36234.8  , 51713.44 , 50057.   , 53761.44 , 28036.47 , 24777.38 ,\n",
            "        20562.73 ],\n",
            "       [28113.84 ,  8658.091, 11236.08 , 20603.29 , 16821.39 , 20444.38 ,\n",
            "        24170.79 , 35524.86 , 39036.26 , 51318.96 , 40860.62 , 26083.74 ,\n",
            "        19876.86 ],\n",
            "       [15281.68 ,  3810.859,  9770.006, 18748.09 , 29311.71 , 22802.36 ,\n",
            "        26635.19 , 28699.58 , 25835.6  , 70428.26 , 30620.16 , 25581.   ,\n",
            "        23701.91 ],\n",
            "       [36766.46 , 10166.14 , 15946.45 , 20468.22 , 44851.83 , 37654.39 ,\n",
            "        28262.85 , 32881.48 , 31197.31 , 41168.02 , 29014.12 , 34697.3  ,\n",
            "        26808.71 ],\n",
            "       [ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251],\n",
            "       [19210.64 ,  9451.407,  7170.096,  6573.493,  7262.504,  8466.096,\n",
            "        10791.6  , 11358.4  , 11268.83 , 11928.65 , 11280.83 , 10306.79 ,\n",
            "         9663.794],\n",
            "       [10238.43 , 10038.21 ,  4539.494,  4773.204,  5569.295,  7163.819,\n",
            "         7795.367,  8738.427,  9528.196, 10172.64 ,  9130.49 ,  8674.652,\n",
            "         8326.857],\n",
            "       [29271.92 , 16966.65 , 12817.22 , 10414.77 , 38556.29 , 13704.49 ,\n",
            "        12345.75 , 10492.11 , 17783.31 , 33074.34 , 21727.96 , 12487.23 ,\n",
            "        12336.3  ],\n",
            "       [ 9113.713,  3189.483,  8561.221, 12143.9  , 20194.84 , 17823.09 ,\n",
            "        19165.6  , 19160.75 , 16129.65 , 13953.47 , 13963.54 , 14566.84 ,\n",
            "        19153.22 ],\n",
            "       [10861.09 ,  3527.603, 10100.51 , 20122.62 , 21703.64 , 18784.44 ,\n",
            "        26654.16 , 35585.44 , 26297.76 , 28429.7  , 29991.59 , 26651.75 ,\n",
            "        26302.86 ],\n",
            "       [13029.19 ,  4096.947, 12660.51 , 17391.69 , 20176.88 , 30939.51 ,\n",
            "        32347.47 , 42654.11 , 62334.09 , 51750.39 , 50261.72 , 32249.87 ,\n",
            "        27588.08 ],\n",
            "       [13747.17 ,  6064.09 , 11660.52 , 44131.81 , 21799.88 , 34285.92 ,\n",
            "        44999.81 , 41557.91 , 30851.62 , 26152.08 , 23482.44 , 18958.37 ,\n",
            "        17940.44 ],\n",
            "       [13024.28 ,  4491.389,  9199.829, 37471.12 , 18161.81 , 26462.47 ,\n",
            "        39266.54 , 45372.93 , 31142.87 , 33532.12 , 24271.95 , 16709.93 ,\n",
            "        15842.14 ],\n",
            "       [52920.68 , 12460.24 , 14539.01 , 70628.47 , 22720.44 , 28388.54 ,\n",
            "        36234.8  , 51713.44 , 50057.   , 53761.44 , 28036.47 , 24777.38 ,\n",
            "        20562.73 ],\n",
            "       [ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251],\n",
            "       [33835.89 , 10039.47 , 10079.46 ,  8033.117, 10461.53 , 10658.41 ,\n",
            "        11425.79 , 11917.09 , 11949.16 , 13566.98 , 11865.49 , 11101.19 ,\n",
            "        10060.95 ],\n",
            "       [10541.11 , 10398.26 , 13323.43 , 20512.11 , 14368.53 , 15074.11 ,\n",
            "        17390.07 , 19810.31 , 17453.21 , 19596.55 , 18918.83 , 20086.91 ,\n",
            "        18725.51 ],\n",
            "       [14362.66 , 18744.05 , 11395.68 , 17604.16 , 12255.51 , 14973.58 ,\n",
            "        18962.06 , 16244.04 , 16262.06 , 13826.58 , 14377.56 , 13519.31 ,\n",
            "        13511.66 ],\n",
            "       [13747.17 ,  6064.09 , 11660.52 , 44131.81 , 21799.88 , 34285.92 ,\n",
            "        44999.81 , 41557.91 , 30851.62 , 26152.08 , 23482.44 , 18958.37 ,\n",
            "        17940.44 ],\n",
            "       [13024.28 ,  4491.389,  9199.829, 37471.12 , 18161.81 , 26462.47 ,\n",
            "        39266.54 , 45372.93 , 31142.87 , 33532.12 , 24271.95 , 16709.93 ,\n",
            "        15842.14 ],\n",
            "       [52920.68 , 12460.24 , 14539.01 , 70628.47 , 22720.44 , 28388.54 ,\n",
            "        36234.8  , 51713.44 , 50057.   , 53761.44 , 28036.47 , 24777.38 ,\n",
            "        20562.73 ],\n",
            "       [ 8118.612,  5110.032, 12845.01 , 15704.74 , 23408.29 , 17766.82 ,\n",
            "        21845.24 , 21889.9  , 64473.39 , 40553.73 , 22041.24 , 19536.16 ,\n",
            "        21640.13 ],\n",
            "       [16555.82 ,  6712.833, 25445.04 , 11565.54 , 29552.94 , 24277.09 ,\n",
            "        27636.95 , 25560.94 , 43225.9  , 40231.53 , 32650.18 , 21682.2  ,\n",
            "        20015.3  ],\n",
            "       [54722.85 , 18249.88 , 29889.41 , 18898.83 , 43132.42 , 31870.11 ,\n",
            "        60596.   , 36680.71 , 36877.51 , 41308.57 , 27701.82 , 22272.7  ,\n",
            "        20371.91 ],\n",
            "       [ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251],\n",
            "       [ 8104.233,  5344.476, 10518.08 , 11515.44 , 11502.12 , 13264.83 ,\n",
            "        18537.86 , 17672.32 , 19276.32 , 19192.07 , 17322.32 , 14392.41 ,\n",
            "        12660.5  ],\n",
            "       [ 1672.14 ,  2563.211,  2475.374,  3410.646,  4425.932,  5530.672,\n",
            "         7249.499,  9209.617, 10143.23 ,  9683.029,  8902.944,  8219.629,\n",
            "         7589.251],\n",
            "       [15756.68 ,  9520.698, 12481.49 , 10016.32 , 11632.07 , 12732.09 ,\n",
            "        15266.26 , 15419.72 , 15986.35 , 15035.07 , 13926.88 , 12311.71 ,\n",
            "        10783.33 ]])}, 'stateTrans': ['sil_0', 'sil_1', 'sil_2', 'z_0', 'z_1', 'z_2', 'iy_0', 'iy_1', 'iy_2', 'r_0', 'r_1', 'r_2', 'ow_0', 'ow_1', 'ow_2', 'sp_0', 'f_0', 'f_1', 'f_2', 'ao_0', 'ao_1', 'ao_2', 'r_0', 'r_1', 'r_2', 'sp_0', 'th_0', 'th_1', 'th_2', 'r_0', 'r_1', 'r_2', 'iy_0', 'iy_1', 'iy_2', 'sp_0', 'sil_0', 'sil_1', 'sil_2'], 'obsloglik': array([[-82.85615934, -77.77520492, -78.78912809, ..., -82.85615934,\n",
            "        -77.77520492, -78.78912809],\n",
            "       [-82.64629092, -75.68935466, -78.4682569 , ..., -82.64629092,\n",
            "        -75.68935466, -78.4682569 ],\n",
            "       [-81.28144137, -74.7891467 , -76.77156599, ..., -81.28144137,\n",
            "        -74.7891467 , -76.77156599],\n",
            "       ...,\n",
            "       [-78.94201025, -88.97211724, -80.73016898, ..., -78.94201025,\n",
            "        -88.97211724, -80.73016898],\n",
            "       [-77.97751766, -86.49199766, -79.0998656 , ..., -77.97751766,\n",
            "        -86.49199766, -79.0998656 ],\n",
            "       [-76.34772424, -81.64468937, -77.4851362 , ..., -76.34772424,\n",
            "        -81.64468937, -77.4851362 ]]), 'viterbiLoglik': -14697.498226298596, 'viterbiPath': array([ 0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
            "        1,  1,  2,  3,  3,  3,  3,  4,  5,  5,  5,  5,  5,  5,  5,  5,  5,\n",
            "        5,  5,  6,  6,  6,  6,  6,  6,  6,  6,  7,  8,  9,  9,  9,  9,  9,\n",
            "        9,  9,  9,  9,  9, 10, 11, 12, 13, 14, 14, 14, 14, 14, 14, 14, 14,\n",
            "       14, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 19, 20, 20,\n",
            "       20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21,\n",
            "       21, 21, 21, 21, 21, 21, 22, 22, 22, 23, 24, 26, 26, 26, 26, 26, 26,\n",
            "       26, 26, 26, 26, 27, 27, 27, 28, 29, 29, 29, 29, 29, 29, 29, 29, 29,\n",
            "       30, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34,\n",
            "       34, 34, 34, 34, 34, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,\n",
            "       36, 36, 36, 36, 36, 36, 37, 38]), 'viterbiStateTrans': ['sil_0', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_1', 'sil_2', 'z_0', 'z_0', 'z_0', 'z_0', 'z_1', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'z_2', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_1', 'iy_2', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_1', 'r_2', 'ow_0', 'ow_1', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'ow_2', 'f_0', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_1', 'f_2', 'ao_0', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_1', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'ao_2', 'r_0', 'r_0', 'r_0', 'r_1', 'r_2', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_0', 'th_1', 'th_1', 'th_1', 'th_2', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_0', 'r_1', 'r_2', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_0', 'iy_1', 'iy_1', 'iy_2', 'iy_2', 'iy_2', 'iy_2', 'iy_2', 'iy_2', 'iy_2', 'iy_2', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_0', 'sil_1', 'sil_2']}\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}